{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import (print_function, absolute_import)\n",
    "\n",
    "import numpy as np\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras import datasets\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import load_model\n",
    "\n",
    "from models import CNN, VGG8\n",
    "from wide_resnet import WideResidualNetwork\n",
    "\n",
    "from keras.callbacks import (\n",
    "    ReduceLROnPlateau,\n",
    "    LearningRateScheduler,\n",
    "    CSVLogger,\n",
    "    EarlyStopping,\n",
    "    ModelCheckpoint)\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "global _SESSION\n",
    "config = tf.ConfigProto(allow_soft_placement=True)\n",
    "config.gpu_options.allow_growth = True\n",
    "_SESSION = tf.Session(config=config)\n",
    "K.set_session(_SESSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_classes = 10\n",
    "batch_size = 128\n",
    "epochs = 200\n",
    "data_augmentation = True\n",
    "checkpoint = None\n",
    "# checkpoint = 'model_checkpoint_cifar10_wide_resnet.h5'\n",
    "# title = 'cifar10_vgg8'\n",
    "title = 'cifar10_wide_resnet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 32, 32, 3)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)                (None, 32, 32, 16)    448                                          \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 32, 32, 16)    64                                           \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 32, 32, 16)    0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)                (None, 32, 32, 160)   23200                                        \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)                (None, 32, 32, 160)   2720                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_1 (Add)                      (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_4 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_5 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_2 (Add)                      (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_6 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_3 (Add)                      (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)                (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)              (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)               (None, 32, 32, 160)   230560                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNorm (None, 32, 32, 160)   640                                          \n",
      "____________________________________________________________________________________________________\n",
      "activation_9 (Activation)        (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_4 (Add)                      (None, 32, 32, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)   (None, 16, 16, 160)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)               (None, 16, 16, 320)   461120                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_10 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)              (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)               (None, 16, 16, 320)   51520                                        \n",
      "____________________________________________________________________________________________________\n",
      "activation_11 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_5 (Add)                      (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_12 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)              (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_13 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_6 (Add)                      (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_14 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)              (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_15 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_7 (Add)                      (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_16 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)              (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)               (None, 16, 16, 320)   921920                                       \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNor (None, 16, 16, 320)   1280                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_17 (Activation)       (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_8 (Add)                      (None, 16, 16, 320)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 8, 8, 320)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)               (None, 8, 8, 640)     1843840                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_18 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)              (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)               (None, 8, 8, 640)     205440                                       \n",
      "____________________________________________________________________________________________________\n",
      "activation_19 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_9 (Add)                      (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_20 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)             (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_21 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_10 (Add)                     (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_22 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)             (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_23 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_11 (Add)                     (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_24 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)             (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)               (None, 8, 8, 640)     3687040                                      \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNor (None, 8, 8, 640)     2560                                         \n",
      "____________________________________________________________________________________________________\n",
      "activation_25 (Activation)       (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "add_12 (Add)                     (None, 8, 8, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePool (None, 1, 1, 640)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)              (None, 640)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 10)            6410                                         \n",
      "====================================================================================================\n",
      "Total params: 36,507,242.0\n",
      "Trainable params: 36,489,290.0\n",
      "Non-trainable params: 17,952.0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = datasets.cifar10.load_data()\n",
    "\n",
    "x_train = x_train.astype(K.floatx())\n",
    "x_test = x_test.astype(K.floatx())\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "if title == 'cifar10_vgg8':\n",
    "    model = VGG8(input_shape=x_train.shape[1:], num_classes=num_classes)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "elif title == 'cifar10_wide_resnet':\n",
    "    model = WideResidualNetwork(depth=28, width=10, dropout_rate=0.3,\n",
    "                                classes=num_classes, include_top=True,\n",
    "                                weights=None)\n",
    "#     sgd = keras.optimizers.SGD(lr=0.1,\n",
    "#                          decay=5e-4,\n",
    "#                          momentum=0.9,\n",
    "#                          nesterov=True)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "if checkpoint is not None:\n",
    "    model = load_model(checkpoint)\n",
    "    \n",
    "\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "checkpointer = ModelCheckpoint(\n",
    "    filepath=\"model_checkpoint_{}.h5\".format(title),\n",
    "    verbose=1,\n",
    "    save_best_only=True)\n",
    "\n",
    "# csvlogger\n",
    "csv_logger = CSVLogger(\n",
    "    'csv_logger_{}.csv'.format(title))\n",
    "# EarlyStopping\n",
    "early_stopper = EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0.001,\n",
    "                              patience=200)\n",
    "# Reduce lr on plateau\n",
    "def schedule(epoch):\n",
    "    lr = K.get_value(sgd.lr)\n",
    "    if epoch in [60, 120, 160]:\n",
    "        lr = lr * 0.8\n",
    "    return lr\n",
    "lr_scheduler = LearningRateScheduler(schedule)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/200\n",
      " 52/390 [===>..........................] - ETA: 233s - loss: 14.5446 - acc: 0.0938 \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
     ]
    }
   ],
   "source": [
    "def normalize(x):\n",
    "    \"\"\"Substract mean and Divide by std.\"\"\"\n",
    "    x -= np.array([125.3, 123.0, 113.9], dtype=K.floatx())\n",
    "    x /= np.array([63.0, 62.1, 66.7], dtype=K.floatx())\n",
    "    return x\n",
    "\n",
    "\n",
    "x_train = normalize(x_train)\n",
    "x_test = normalize(x_test)\n",
    "\n",
    "if not data_augmentation:\n",
    "    print('No data augmentation applied.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=[csv_logger, checkpointer, early_stopper]) \n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally (fraction of total width)\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically (fraction of total height)\n",
    "        height_shift_range=0.1,\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "    # Compute quantities required for feature-wise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train,\n",
    "                                     batch_size=batch_size),\n",
    "                        steps_per_epoch=x_train.shape[0] // batch_size,\n",
    "                        epochs=epochs,\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        callbacks=[csv_logger, checkpointer, early_stopper])\n",
    "    model.save('{}.h5'.format(title))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
